
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "59"
csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H6C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 49 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_
Found 47 auto_metrics notebooks in /home/obs/src/H6C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459864	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	50.450052	6.842178	2.077257	-1.089873	3.677281	0.449751	5.471731	12.050349	0.6343	0.6813	0.3855	nan	nan
2459863	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	24.406391	5.381905	-0.810319	0.508891	5.804978	3.713988	19.278061	18.066216	0.6490	0.6660	0.3764	nan	nan
2459862	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	2.631815	10.328683	0.676779	0.657894	4.714494	3.801800	21.662981	14.692303	0.6836	0.6915	0.3902	nan	nan
2459861	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	25.855451	1.532222	-0.720041	0.810004	0.120019	1.529031	2.064155	3.220936	0.6349	0.6900	0.3906	nan	nan
2459860	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	21.904884	2.282970	1.054804	0.774635	7.286083	0.971598	3.163818	2.950127	0.6686	0.6839	0.3832	nan	nan
2459859	digital_maintenance	100.00%	0.00%	0.00%	0.00%	-	-	20.057258	1.844305	-0.742431	1.024146	-0.010641	1.563357	5.703602	7.834898	0.6635	0.6850	0.3827	nan	nan
2459858	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	27.201683	1.551202	-0.791642	0.865188	0.291481	2.403911	3.708576	5.056602	0.6547	0.6945	0.3951	2.483735	2.420656
2459857	digital_maintenance	0.00%	100.00%	100.00%	0.00%	-	-	2.431043	-0.630875	-1.325867	-0.642617	1.134683	0.733186	-0.612285	1.017405	0.0288	0.0266	0.0013	nan	nan
2459856	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	36.872522	4.123114	1.390223	1.187818	2.715057	0.874903	4.353870	6.058571	0.6456	0.7049	0.3876	2.617801	2.510649
2459855	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	36.769955	5.263141	1.362815	0.624491	1.049044	1.398526	2.532185	3.828587	0.6363	0.7165	0.4213	2.664733	2.585901
2459854	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	43.721341	4.703118	1.820185	0.095387	0.915799	0.913826	1.051646	1.714877	0.6480	0.7450	0.4263	2.443960	2.630338
2459853	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	30.645108	3.140462	2.434389	0.250182	2.991383	0.566680	4.194870	5.984501	0.6741	0.6953	0.3995	2.703891	2.743787
2459852	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	19.368184	3.246582	2.186928	0.127206	5.347286	0.468716	5.050586	0.591528	0.7842	0.8461	0.2432	5.201215	4.633717
2459851	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	21.408526	1.992630	2.651270	0.392886	7.762365	1.898464	5.350345	3.116609	0.7071	0.7542	0.3358	2.799971	2.907872
2459850	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	26.038955	3.197912	2.226979	0.241278	3.710470	0.146991	2.885598	2.037742	0.0457	0.0488	0.0014	1.182413	1.178893
2459849	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	38.165788	3.916434	5.994384	1.500904	3.409707	1.421007	6.934457	9.013549	0.0469	0.0544	0.0014	1.173547	1.169185
2459848	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	24.505006	3.267827	1.917040	2.352809	3.438660	-0.004557	4.790111	4.921548	0.0508	0.0543	0.0018	1.201411	1.201962
2459847	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	34.772840	3.457578	2.228304	1.715500	4.486032	1.567689	5.637892	4.807589	0.0408	0.0492	0.0015	1.159278	1.157740
2459846	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	27.210792	3.531416	1.010270	1.399444	4.948944	2.089112	4.175592	3.060530	0.0527	0.0555	0.0026	1.169073	1.178312
2459845	digital_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	23.526629	5.538594	2.335410	2.737583	3.733709	1.254090	22.658554	10.241235	0.7171	0.7530	0.3493	4.915205	5.549201
2459844	digital_maintenance	100.00%	100.00%	100.00%	0.00%	-	-	4.262356	1.219994	6.765837	1.487968	1.107066	1.292099	-0.687745	0.549580	0.0288	0.0259	0.0021	nan	nan
2459843	digital_maintenance	100.00%	0.66%	0.66%	0.00%	100.00%	0.00%	1.270081	19.158184	10.781159	-0.029407	3.770363	1.333786	4.581501	24.254117	0.7635	0.7336	0.3715	4.834336	3.320700

Antenna Season Report Notebook¶

Antenna 59 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 59, Node N05:

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 59: 2459864

Antenna 59: 2459863

Antenna 59: 2459862

Antenna 59: 2459861

Antenna 59: 2459860

Antenna 59: 2459859

Antenna 59: 2459858

Antenna 59: 2459857

Antenna 59: 2459856

Antenna 59: 2459855

Antenna 59: 2459854

Antenna 59: 2459853

Antenna 59: 2459852

Antenna 59: 2459851

Antenna 59: 2459850

Antenna 59: 2459849

Antenna 59: 2459848

Antenna 59: 2459847

Antenna 59: 2459846

Antenna 59: 2459845

Antenna 59: 2459844

Antenna 59: 2459843

Antenna Season Report Notebook¶

Antenna 59 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 59, Node N05:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 59: 2459864

Antenna 59: 2459863

Antenna 59: 2459862

Antenna 59: 2459861

Antenna 59: 2459860

Antenna 59: 2459859

Antenna 59: 2459858

Antenna 59: 2459857

Antenna 59: 2459856

Antenna 59: 2459855

Antenna 59: 2459854

Antenna 59: 2459853

Antenna 59: 2459852

Antenna 59: 2459851

Antenna 59: 2459850

Antenna 59: 2459849

Antenna 59: 2459848

Antenna 59: 2459847

Antenna 59: 2459846

Antenna 59: 2459845

Antenna 59: 2459844

Antenna 59: 2459843

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶